using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Text.Encodings.Web;
using System.Text.Unicode;
using System.Threading.Tasks;
using AngleSharp;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;

namespace WebUrlCollector.Controllers
{
    [ApiController]
    [Route("api/[controller]")]
    public class UrlCollectorController : ControllerBase
    {
        private readonly HttpClient _httpClient;
        private readonly ILogger<UrlCollectorController> _logger;

        public UrlCollectorController(IHttpClientFactory httpClientFactory, ILogger<UrlCollectorController> logger)
        {
            _httpClient = httpClientFactory.CreateClient("BaiduClient");
            _logger = logger;
        }

        // 用于实时更新的Server-Sent Events端点
        [HttpGet("collect")]
        public async Task CollectUrlsSse(string keyword, int pages, string cookie = "")
        {
            Response.Headers.Add("Content-Type", "text/event-stream");
            Response.Headers.Add("Cache-Control", "no-cache");
            Response.Headers.Add("Connection", "keep-alive");
            
            if (string.IsNullOrWhiteSpace(keyword))
            {
                await SendEventAsync(new { status = "error", message = "关键字不能为空" });
                return;
            }
            
            if (pages <= 0 || pages > 100)
            {
                await SendEventAsync(new { status = "error", message = "页数必须在1-100之间" });
                return;
            }
            
            var urls = new HashSet<string>();
            var totalPages = pages;
            
            // 发送开始日志
            await SendEventAsync(new { log = $"开始采集URL，关键字: {keyword}，页数: {totalPages}" });
            
            try
            {
                for (int page = 0; page < totalPages; page++)
                {
                    // 检查客户端是否断开连接
                    if (HttpContext.RequestAborted.IsCancellationRequested)
                    {
                        await SendEventAsync(new { log = "采集任务已取消" });
                        return;
                    }
                    
                    var currentPage = page + 1;
                    await SendEventAsync(new { log = $"正在采集第 {currentPage} 页..." });
                    
                    try
                    {
                        // 构建百度搜索URL
                        var searchUrl = BuildBaiduSearchUrl(keyword, page);
                        await SendEventAsync(new { log = $"搜索URL: {searchUrl}" });
                        
                        // 发送请求获取搜索结果
                        var html = await FetchSearchResults(searchUrl, cookie);
                        
                        // 检查是否遇到验证码
                        if (IsVerificationPage(html))
                        {
                            await SendEventAsync(new { log = "警告: 遇到百度验证码，请提供有效的Cookie或稍后再试" });
                            continue; // 跳过当前页面，尝试下一页
                        }
                        
                        // 提取URL
                        var pageUrls = ExtractUrlsFromHtml(html);
                        
                        // 添加到结果集（自动去重）
                        foreach (var url in pageUrls)
                        {
                            urls.Add(url);
                        }
                        
                        await SendEventAsync(new { log = $"第 {currentPage} 页采集完成，提取到 {pageUrls.Count} 个URL" });
                        
                        // 实时发送当前结果和进度
                        await SendEventAsync(new {
                            urls = new List<string>(urls),
                            progress = (int)((page + 1) * 100.0 / totalPages)
                        });
                        
                        // 添加延迟以避免反爬
                        if (page < totalPages - 1)
                        {
                            var delayTime = new Random().Next(3000, 6000); // 3-6秒随机延迟
                            await SendEventAsync(new { log = $"添加反爬延迟 {delayTime}ms..." });
                            await Task.Delay(delayTime);
                        }
                    }
                    catch (Exception ex)
                    {
                        await SendEventAsync(new { log = $"第 {currentPage} 页采集失败: {ex.Message}" });
                        _logger.LogError(ex, $"采集第{currentPage}页失败");
                    }
                }
                
                await SendEventAsync(new { log = $"采集完成，总共获取到 {urls.Count} 个唯一域名" });
                await SendEventAsync(new {
                    urls = new List<string>(urls),
                    progress = 100,
                    status = "completed"
                });
            }
            catch (Exception ex)
            {
                await SendEventAsync(new { log = $"严重错误: {ex.Message}" });
                _logger.LogError(ex, "采集过程中发生严重错误");
                await SendEventAsync(new { status = "error", message = ex.Message });
            }
        }
        
        // 用于发送Server-Sent Events的辅助方法
        private async Task SendEventAsync(object data)
        {
            var json = System.Text.Json.JsonSerializer.Serialize(data);
            await Response.WriteAsync($"data: {json}\n\n");
            await Response.Body.FlushAsync();
        }
        
        // 保留原有的POST端点用于向后兼容
        [HttpPost("collect")]
        public async Task<IActionResult> CollectUrls([FromBody] CollectRequest request)
        {
            if (string.IsNullOrWhiteSpace(request.Keyword))
            {
                return BadRequest(new { error = "关键字不能为空" });
            }

            if (request.Pages <= 0 || request.Pages > 100)
            {
                return BadRequest(new { error = "页数必须在1-100之间" });
            }

            var logs = new List<string>();
            var urls = new HashSet<string>(); // 使用HashSet去重
            var totalPages = request.Pages;
            
            logs.Add($"开始采集URL，关键字: {request.Keyword}，页数: {totalPages}");
            
            try
            {
                for (int page = 0; page < totalPages; page++)
                {
                    var currentPage = page + 1;
                    logs.Add($"正在采集第 {currentPage} 页...");
                    
                    try
                    {
                        // 构建百度搜索URL
                        var searchUrl = BuildBaiduSearchUrl(request.Keyword, page);
                        logs.Add($"搜索URL: {searchUrl}");
                        
                        // 发送请求获取搜索结果
                        var html = await FetchSearchResults(searchUrl, request.Cookie);
                        
                        // 检查是否遇到验证码
                        if (IsVerificationPage(html))
                        {
                            logs.Add("警告: 遇到百度验证码，请提供有效的Cookie或稍后再试");
                            continue; // 跳过当前页面，尝试下一页
                        }
                        
                        // 提取URL
                        var pageUrls = ExtractUrlsFromHtml(html);
                        
                        // 添加到结果集（自动去重）
                        foreach (var url in pageUrls)
                        {
                            urls.Add(url);
                        }
                        
                        logs.Add($"第 {currentPage} 页采集完成，提取到 {pageUrls.Count} 个URL");
                        
                        // 添加延迟以避免反爬
                        if (page < totalPages - 1)
                        {
                            var delayTime = new Random().Next(3000, 6000); // 3-6秒随机延迟
                            logs.Add($"添加反爬延迟 {delayTime}ms...");
                            await Task.Delay(delayTime);
                        }
                    }
                    catch (Exception ex)
                    {
                        logs.Add($"第 {currentPage} 页采集失败: {ex.Message}");
                        _logger.LogError(ex, $"采集第{currentPage}页失败");
                    }
                }
                
                logs.Add($"采集完成，总共获取到 {urls.Count} 个唯一URL");
                
                return Ok(new {
                    urls = new List<string>(urls),
                    logs = logs
                });
            }
            catch (Exception ex)
            {
                logs.Add($"严重错误: {ex.Message}");
                _logger.LogError(ex, "采集过程中发生严重错误");
                return StatusCode(500, new { error = "采集过程中发生错误", logs = logs });
            }
        }

        private string BuildBaiduSearchUrl(string keyword, int page)
        {
            // 百度搜索URL构造，每页10条结果
            int start = page * 10;
            var encoder = UrlEncoder.Create(UnicodeRanges.All);
            return $"https://www.baidu.com/s?wd={encoder.Encode(keyword)}&pn={start}";
        }

        private async Task<string> FetchSearchResults(string url, string cookie)
        {
            // 配置请求头
            var request = new HttpRequestMessage(HttpMethod.Get, url);
            
            // 添加Cookie（如果提供）
            if (!string.IsNullOrEmpty(cookie))
            {
                request.Headers.Add("Cookie", cookie);
            }
            
            // 添加其他常见请求头
            request.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8");
            request.Headers.Add("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
            request.Headers.Add("Connection", "keep-alive");
            // 暂时不请求压缩，避免解压问题
            // request.Headers.Add("Accept-Encoding", "gzip, deflate, br");
            
            // 添加用户代理
            request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0");
            
            // 发送请求
            var response = await _httpClient.SendAsync(request);
            response.EnsureSuccessStatusCode();
            
            // 读取响应内容
            return await response.Content.ReadAsStringAsync();
        }

        private bool IsVerificationPage(string html)
        {
            // 检查是否包含验证码相关关键词
            return html.Contains("验证码") && 
                   (html.Contains("请输入验证码") || html.Contains("validate-form") || html.Contains("verifycode"));
        }

        private List<string> ExtractUrlsFromHtml(string html)
        {
            var urls = new List<string>();
            
            try
            {
                _logger.LogInformation("开始解析HTML，长度: {Length}", html.Length);
                
                // 首先检查HTML是否有效
                if (string.IsNullOrEmpty(html) || html.Length < 100)
                {
                    _logger.LogWarning("HTML内容异常，长度过短");
                    return urls;
                }
                
                // 打印HTML前100个字符用于调试
                _logger.LogDebug("HTML预览: {Preview}", html.Substring(0, Math.Min(100, html.Length)));
                
                // 使用AngleSharp解析HTML
                var parser = new HtmlParser();
                var document = parser.ParseDocument(html);
                
                // 策略1: 提取所有结果链接（桌面版 - 更新选择器）
                _logger.LogInformation("尝试使用CSS选择器提取链接...");
                var selectors = new[] {
                    "a[tpl='se_com_default']",  // 原始选择器
                    "h3.t a",                    // 常见的标题链接选择器
                    "div.result h3 a",          // 结果标题链接
                    "div.result-op a",          // 优化结果链接
                    "div.c-container a"
                };
                
                foreach (var selector in selectors)
                {
                    var results = document.QuerySelectorAll(selector);
                    _logger.LogInformation("选择器 {Selector} 找到 {Count} 个元素", selector, results.Length);
                    foreach (var element in results)
                    {
                        var href = element.GetAttribute("href");
                        ProcessUrl(href, urls);
                    }
                }
                
                // 策略2: 使用正则表达式提取URL（备用）
                _logger.LogInformation("尝试使用正则表达式提取链接...");
                var regex = new Regex("href=[\"'](https?://[^\"']+)[\"']", RegexOptions.IgnoreCase);
                var matches = regex.Matches(html);
                _logger.LogInformation("正则表达式找到 {Count} 个匹配", matches.Count);
                foreach (Match match in matches)
                {
                    if (match.Groups.Count > 1)
                    {
                        ProcessUrl(match.Groups[1].Value, urls);
                    }
                }
                
                // 策略3: 备用正则表达式（简化版）
                var simpleRegex = new Regex("https?://\\S+");
                var simpleMatches = simpleRegex.Matches(html);
                _logger.LogInformation("简化正则找到 {Count} 个匹配", simpleMatches.Count);
                foreach (Match match in simpleMatches)
                {
                    ProcessUrl(match.Value, urls);
                }
                
                _logger.LogInformation("HTML解析完成，共提取 {Count} 个URL", urls.Count);
            }
            catch (Exception ex)
            {
                // 解析失败时记录错误但不影响整体流程
                _logger.LogError(ex, "HTML解析错误");
            }
            
            return urls;
        }
        
        private void ProcessUrl(string href, ICollection<string> urls)
        {
            if (!string.IsNullOrEmpty(href))
            {
                // 处理百度的重定向链接
                var cleanUrl = CleanBaiduUrl(href);
                if (!string.IsNullOrEmpty(cleanUrl) && IsValidExternalLink(cleanUrl))
                {
                    // 提取域名部分
                    var domain = ExtractDomainFromUrl(cleanUrl);
                    // 过滤掉.gov.cn和.edu.cn域名，并检查重复
                    if (!string.IsNullOrEmpty(domain) && 
                        !domain.EndsWith(".gov.cn") && 
                        !domain.EndsWith(".edu.cn") && 
                        !urls.Contains(domain))
                    {
                        urls.Add(domain);
                        _logger.LogDebug("添加域名: {Domain}", domain);
                    }
                    else if (!string.IsNullOrEmpty(domain) && 
                             (domain.EndsWith(".gov.cn") || domain.EndsWith(".edu.cn")))
                    {
                        _logger.LogDebug("过滤域名: {Domain} (政府或教育域名)", domain);
                    }
                    else if (!string.IsNullOrEmpty(domain) && urls.Contains(domain))
                    {
                        _logger.LogDebug("跳过重复域名: {Domain}", domain);
                    }
                }
            }
        }
        
        private string ExtractDomainFromUrl(string url)
        {
            try
            {
                // 解析URL获取域名
                var uri = new Uri(url);
                return uri.Host;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "提取域名失败: {Url}", url);
                return null;
            }
        }

        private string CleanBaiduUrl(string url)
        {
            try
            {
                // 如果是百度的重定向链接，尝试提取真实URL
                if (url.StartsWith("https://www.baidu.com/link?"))
                {
                    // 从URL参数中提取真实链接
                    var uri = new Uri(url);
                    var query = uri.Query;
                    
                    // 尝试多种参数格式
                    string[] paramNames = { "url=", "wd=", "q=" };
                    foreach (var param in paramNames)
                    {
                        if (query.Contains(param))
                        {
                            int startIndex = query.IndexOf(param) + param.Length;
                            int endIndex = query.IndexOf("&", startIndex);
                            if (endIndex == -1) endIndex = query.Length;
                            
                            var encodedUrl = query.Substring(startIndex, endIndex - startIndex);
                            // 解码URL
                            try 
                            { 
                                var decodedUrl = Uri.UnescapeDataString(encodedUrl);
                                // 再次解码（百度可能进行了双重编码）
                                try {
                                    decodedUrl = Uri.UnescapeDataString(decodedUrl);
                                } catch {}
                                return decodedUrl;
                            } 
                            catch 
                            { 
                                // 解码失败，继续尝试其他参数格式
                            }
                        }
                    }
                }
                
                // 直接检查是否是百度重定向链接，百度可能使用不同的格式
                if (url.Contains("www.baidu.com/link?"))
                {
                    // 简单处理：直接返回链接本身，让调用者决定
                    return url;
                }
                
                // 如果URL本身就是正常的外部链接，直接返回
                if (url.StartsWith("http://") || url.StartsWith("https://"))
                {
                    return url;
                }
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "清理百度URL失败: {url}", url);
                // URL解析失败，返回原值
            }
            
            return url;
        }

        private bool IsValidExternalLink(string url)
        {
            // 过滤掉百度内部链接和无效链接
            return !url.Contains("baidu.com") && 
                   !url.Contains("baidu.cn") && 
                   !url.StartsWith("javascript:") && 
                   !url.StartsWith("#") && 
                   !url.StartsWith("mailto:") && 
                   !url.StartsWith("tel:");
        }
    }

    public class CollectRequest
    {
        public string Keyword { get; set; } = string.Empty;
        public int Pages { get; set; } = 5;
        public string Cookie { get; set; } = string.Empty;
    }
}